package com.wxgo.utils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.jsoup.Connection;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import com.jfinal.plugin.activerecord.Db;
import com.jfinal.plugin.activerecord.Record;

public class Spider {

	public static boolean catchAgent(String url) {
		List<Record> rList = new ArrayList<Record>();
		Record r = null;
		boolean flag = false;
		try {
			Document doc = getDocByURL(url);
			if (doc != null) {
				Elements trs = doc.select("table").select("tr");
				int i;
				String timelen = null;
				for (i = 0; i < trs.size(); i++) {
					Elements tds = trs.get(i).select("td");
					if (tds.size() < 6) {
						continue;
					}
					r = new Record();
					String ip = tds.get(1).text();
					long count = Db.queryLong("select count(*) from agent where ip=?", ip);
					if (count > 0)
						continue;
					timelen = tds.get(6).text();
					if (timelen.contains("小时") || timelen.contains("分钟") || timelen.equals("1天"))
						continue;
					r.set("ip", ip);
					r.set("port", Integer.parseInt(tds.get(2).text()));
					r.set("address", tds.get(3).text());
					r.set("isanony", tds.get(4).text());
					r.set("type", tds.get(5).text());
					// timelen = StringUtils.getNumbers(timelen);

					r.set("timelen", timelen);
					rList.add(r);
				}
				System.out.println("*****本次获取新IP数量为" + rList.size() + "个******");
				int[] nums = Db.batchSave("agent", rList, rList.size());
				flag = nums.length > 0;
			} else {

			}
		} catch (IOException e) {
			e.printStackTrace();
		}
		return flag;
	}

	/**
	 * 根据URL获得Document
	 */
	public static Document getDocByURL(String url) throws IOException {
		Document doc = null;
		Connection conn = Jsoup.connect(url).ignoreContentType(true).ignoreHttpErrors(true)
				.userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0")
				.header("Connection", "keep-alive").header("Upgrade-Insecure-Requests", "1");

		Response resp = conn.execute();
		int code = resp.statusCode();
		if (code == 200 || code == 503) {
			doc = conn.get();
			return doc;
		}
		return doc;

	}

	public static void main(String[] args) {
		Spider.catchAgent("http://www.xicidaili.com");
		// Spider.catchAgent("http://www.kuaidaili.com/free/inha/");
	}
}
